﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Net;
using System.Threading;
using System.IO;
using System.Text.RegularExpressions;

namespace SimpleCrawler
{
    public partial class Form1 : Form
    {
        private List<string> urls = new List<String>();
        private List<string> newUrls = new List<string>();
        public List<string> totalUrls = new List<string>();

        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            if (textBox1.Text.Trim() == "")
            {
                return; 
            }
            totalUrls.Clear();
            urls.Clear();
            newUrls.Clear();

            Match match = Regex.Match(textBox1.Text, "^.+://");
            if (!match.Success)
            {
                // use http as default protocol
                urls.Add("http://" + textBox1.Text);
            }
            else
            {
                urls.Add(textBox1.Text);
            }
            totalUrls.Add(textBox1.Text);
            richTextBox1.Clear();
            timer1.Start();
            button1.Enabled = false;
            button2.Enabled = true;
            generalStatsInfoToolStripMenuItem.Enabled = false;

            checkBox1.Enabled = false;
            checkBox2.Enabled = false;
            checkBox3.Enabled = false;
            textBox2.Enabled = false;
            textBox3.Enabled = false;
        }

        private void button2_Click(object sender, EventArgs e)
        {
            label2.Text = "Not Started";
            label3.Text = "0 URLs Found";
            stop();
        }

        private void stop()
        {
            timer1.Stop();

            button1.Enabled = true;
            button2.Enabled = false;
            generalStatsInfoToolStripMenuItem.Enabled = true;

            checkBox1.Enabled = true;
            checkBox2.Enabled = true;
            checkBox3.Enabled = true;
            textBox2.Enabled = true;
            textBox3.Enabled = true;

            // backgroundWorker1.CancelAsync();
        }

        private void timer1_Tick(object sender, EventArgs e)
        {
            if (backgroundWorker1.IsBusy)
                return;

            if (urls.Count == 0)
            {
                richTextBox1.AppendText("No more urls to crawl\n");
                stop();
                return;
            }
            label2.Text = "Crawling " + urls[0];
            backgroundWorker1.RunWorkerAsync();
        }

        private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
        {
            try
            {
                string curl = urls[0];
                HttpWebRequest wrq = (HttpWebRequest)HttpWebRequest.Create(curl);
                urls.RemoveAt(0);
                wrq.AllowAutoRedirect = true;
                HttpWebResponse wrs = (HttpWebResponse)wrq.GetResponse();

                if (wrs.StatusCode == HttpStatusCode.OK)
                {
                    // parse hyperlink
                    StreamReader sr = new StreamReader(wrs.GetResponseStream(), System.Text.Encoding.GetEncoding(wrs.CharacterSet));
                    string str = sr.ReadToEnd().ToLower();

                    // check charset in web page
                    Match charsetMatch = Regex.Match(str, "<meta.+charset=\"{0,1}([^\"]+)\"", RegexOptions.IgnoreCase);
                    if (charsetMatch.Success && charsetMatch.Groups[1].Value.ToLower() != wrs.CharacterSet.ToLower())
                    {
                        wrq = (HttpWebRequest)HttpWebRequest.Create(curl);
                        wrs = (HttpWebResponse)wrq.GetResponse();
                        sr = new StreamReader(wrs.GetResponseStream(), System.Text.Encoding.GetEncoding(charsetMatch.Groups[1].Value));
                        str = sr.ReadToEnd().ToLower();
                    }

                    string up1 = "<a[^>]*href=(http\\S+)\\s[^>]*>(((?!</a>).)*)</a>";
                    string up2 = "<a[^>]*href=\"(http[^\"]+)\"[^>]*>(((?!</a>).)*)</a>";

                    while (true)
                    {
                        int is1 = -1, is2 = -1, fs = -1;
                        int l = -1;
                        string url = "", anchor = ""; 
                        Match m1, m2;
                        m1 = Regex.Match(str, up1);
                        m2 = Regex.Match(str, up2);

                        if (m1.Success) is1 = m1.Index;
                        if (m2.Success) is2 = m2.Index;

                        if (is1 == -1 && is2 == -1)
                            break;

                        if (is1 != -1) { fs = is1; l = m1.Length; url = m1.Groups[1].Value; anchor = m1.Groups[2].Value; }
                        if (is2 != -1 && (fs == -1 || is2 < fs)) { fs = is2; l = m2.Length; url = m2.Groups[1].Value; anchor = m2.Groups[2].Value; }

                        if ((url.Trim().ToLower().IndexOf("http://") == 0 || url.Trim().ToLower().IndexOf("https://") == 0) && totalUrls.IndexOf(url) == -1)
                        {
                            if (url == "http://video.sina.com.cn/vlist/news/zt/topvideo20130108/?opsubject_id=top1#123736150")
                            {
                                Console.WriteLine("test");
                            }
                            urls.Add(url);
                            anchor = anchor.Trim();
                            if (anchor.IndexOf("<img") != -1 || anchor.IndexOf("<div") != -1)
                                anchor = "";

                            Match match = Regex.Match(anchor, "span.*>\\s*([\\S ]*)\\s*</span");
                            
                            if (match.Success)
                            {
                                anchor = match.Groups[1].Value;
                            }

                            if (anchor != "")
                                anchor += " ";

                            bool cont = true;

                            // check domain
                            if (checkBox1.Checked)
                            {
                                string reg = "http[a-z0-9:/.]+" + textBox2.Text;
                                Match dm = Regex.Match(url, @reg);
                                if (!dm.Success)
                                    cont = false;
                            }

                            if (cont)
                            {
                                if (checkBox3.Checked)
                                    newUrls.Add(anchor + url);
                                else
                                    newUrls.Add(url);
                                totalUrls.Add(url);
                            }
                        }


                        str = str.Substring(l);
                    }

                    /*
                    int iStart = str.IndexOf("<a href=\"");
                    while (iStart != -1)
                    {
                        iStart += 9;
                        int iEnd = str.IndexOf("\"", iStart);
                        string url = str.Substring(iStart, iEnd - iStart);

                        string anchor = str.Substring(str.IndexOf(">", iEnd) + 1, (str.IndexOf("</a>", iEnd) - str.IndexOf(">", iEnd) - 1));
                        if ((url.Trim().ToLower().IndexOf("http://") == 0 || url.Trim().ToLower().IndexOf("https://") == 0) && totalUrls.IndexOf(url) == -1)
                        {
                            urls.Add(url);
                            anchor = anchor.Trim();
                            if (anchor.IndexOf("<img") != -1 || anchor.IndexOf("<div") != -1)
                                anchor = "";

                            Match match = Regex.Match(anchor, "span.*>\\s*([\\S ]*)\\s*</span");
                            if (match.Success)
                            {
                                anchor = match.Groups[1].Value;
                            }

                            if (anchor != "")
                                anchor += " ";

                            bool cont = true;

                            // check domain
                            if (checkBox1.Checked)
                            {
                                string reg = "http[a-z0-9:/.]+" + textBox2.Text;
                                Match dm = Regex.Match(url, @reg);
                                if (!dm.Success)
                                    cont = false;
                            }

                            if (cont)
                            {
                                if (checkBox3.Checked)
                                    newUrls.Add(anchor + url);
                                else
                                    newUrls.Add(url);
                                totalUrls.Add(url);
                            }
                        }
                        iStart = str.IndexOf("<a href=\"", iEnd);
                    }
                    */
                }
            }
            catch (Exception)
            {

            }
        }

        private void backgroundWorker1_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
        {
            for (int i = 0; i < newUrls.Count; i++)
            {
                if (checkBox2.Checked && textBox3.Text.Trim() != "")
                {
                    if (newUrls[i].IndexOf(textBox3.Text.Trim()) == -1)
                        continue;
                }

                int pe = richTextBox1.Text.Length; 

                richTextBox1.AppendText(newUrls[i] + "\n");
                if (checkBox3.Checked)
                {   
                    // check archor position
                    int ie = newUrls[i].ToLower().Trim().IndexOf("http");
                    if (ie > 0)
                    {
                        richTextBox1.Select(pe, ie);
                        richTextBox1.SelectionColor = Color.Green;
                    }
                }
            }
            newUrls.Clear();
            label3.Text = totalUrls.Count + " URLs Found";
        }

        private void generalStatsInfoToolStripMenuItem_Click(object sender, EventArgs e)
        {
            Form2 statsForm = new Form2(this);
            statsForm.StartPosition = FormStartPosition.CenterParent;
            statsForm.ShowDialog();
        }

        private void domainsToolStripMenuItem_Click(object sender, EventArgs e)
        {
            Form3 domainStat = new Form3(this);
            domainStat.ShowDialog();
        }

        private void checkBox1_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBox1.Checked)
            {
                textBox2.Enabled = true;
            }
            else
            {
                textBox2.Enabled = false;
            }
        }

        private void checkBox2_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBox2.Checked)
            {
                textBox3.Enabled = true;
            }
            else
            {
                textBox3.Enabled = false;
            }
        }
    }
}
